{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "67986540-d293-4ed7-a724-2c775478e219",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:25:11.724728Z",
     "iopub.status.busy": "2022-10-13T10:25:11.724337Z",
     "iopub.status.idle": "2022-10-13T10:25:12.437208Z",
     "shell.execute_reply": "2022-10-13T10:25:12.436311Z",
     "shell.execute_reply.started": "2022-10-13T10:25:11.724669Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                           PRE amazon_fashion_interaction.100K.extra.parquet/\n",
      "                           PRE amazon_fashion_interaction.100K.parquet/\n",
      "                           PRE amazon_fashion_interaction.csv/\n",
      "                           PRE amazon_fashion_interaction.parquet/\n",
      "                           PRE amazon_fashion_interaction.small.parquet/\n",
      "                           PRE amazon_fashion_item.100K.extra.parquet/\n",
      "                           PRE amazon_fashion_item.100K.parquet/\n",
      "                           PRE amazon_fashion_item.csv/\n",
      "                           PRE amazon_fashion_item.parquet/\n",
      "                           PRE amazon_fashion_item.small.parquet/\n",
      "                           PRE amazon_fashion_user.100K.extra.parquet/\n",
      "                           PRE amazon_fashion_user.100K.parquet/\n",
      "                           PRE amazon_fashion_user.csv/\n",
      "                           PRE amazon_fashion_user.parquet/\n",
      "                           PRE amazon_fashion_user.small.parquet/\n",
      "2022-09-08 14:11:37 3247339254 amazon_fashion_product.json\n",
      "2022-09-08 14:13:55 3059298908 amazon_fashion_review.json\n"
     ]
    }
   ],
   "source": [
    "!aws s3 ls ${MY_S3_BUCKET}/sass/ecommerce/amazonfashion/dataset/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "eb4eb50d-5f1c-4dbf-a796-cdd934edd2cb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:25:12.439011Z",
     "iopub.status.busy": "2022-10-13T10:25:12.438665Z",
     "iopub.status.idle": "2022-10-13T10:25:13.219554Z",
     "shell.execute_reply": "2022-10-13T10:25:13.218591Z",
     "shell.execute_reply.started": "2022-10-13T10:25:12.438981Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import subprocess\n",
    "import yaml\n",
    "import argparse\n",
    "import sys \n",
    "import importlib\n",
    "import metaspore as ms\n",
    "import pandas as pd\n",
    "import warnings\n",
    "import pyspark.sql.functions as F\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "pd.set_option('display.max_rows', None)\n",
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "42383785-0d8a-453c-b7ef-e68deb7b2a89",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:25:13.220934Z",
     "iopub.status.busy": "2022-10-13T10:25:13.220607Z",
     "iopub.status.idle": "2022-10-13T10:25:13.294752Z",
     "shell.execute_reply": "2022-10-13T10:25:13.293986Z",
     "shell.execute_reply.started": "2022-10-13T10:25:13.220903Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def init_spark(local, app_name, batch_size, worker_count, server_count,\n",
    "               worker_memory, server_memory, coordinator_memory, **kwargs):\n",
    "    subprocess.run(['zip', '-r', os.getcwd() + '/python.zip', 'python'], cwd='../../../')\n",
    "    spark_confs={\n",
    "        \"spark.kubernetes.namespace\":\"${MY_NAMESPACE}\",\n",
    "        \"spark.network.timeout\":\"500\",\n",
    "        \"spark.submit.pyFiles\":\"python.zip\",\n",
    "        \"spark.ui.showConsoleProgress\": \"true\",\n",
    "        \"spark.kubernetes.executor.deleteOnTermination\":\"true\",\n",
    "    }\n",
    "    spark = ms.spark.get_session(\n",
    "        local=local,\n",
    "        app_name=app_name,\n",
    "        batch_size=batch_size,\n",
    "        worker_count=worker_count,\n",
    "        server_count=server_count,\n",
    "        worker_memory=worker_memory,\n",
    "        server_memory=server_memory,\n",
    "        coordinator_memory=coordinator_memory,\n",
    "        spark_confs=spark_confs)\n",
    "    sc = spark.sparkContext\n",
    "    print('Debug -- spark init')\n",
    "    print('Debug -- version:', sc.version)   \n",
    "    print('Debug -- applicaitonId:', sc.applicationId)\n",
    "    print('Debug -- uiWebUrl:', sc.uiWebUrl)\n",
    "    return spark\n",
    "\n",
    "def stop_spark(spark):\n",
    "    print('Debug -- spark stop')\n",
    "    spark.sparkContext.stop()\n",
    "\n",
    "def read_dataset(spark, dataset_path):\n",
    "    dataset = spark.read.json(dataset_path)\n",
    "    print('Debug -- item dataset count:', dataset.count())\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fe30d2f-4aec-43b5-a0c8-70ad1461facb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "33852308-a93e-4e0e-9df4-f7cf06a3a7df",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:25:13.296461Z",
     "iopub.status.busy": "2022-10-13T10:25:13.296243Z",
     "iopub.status.idle": "2022-10-13T10:29:48.084456Z",
     "shell.execute_reply": "2022-10-13T10:29:48.083619Z",
     "shell.execute_reply.started": "2022-10-13T10:25:13.296443Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "updating: python/ (stored 0%)\n",
      "updating: python/algos/ (stored 0%)\n",
      "updating: python/algos/deepfm_net.py (deflated 69%)\n",
      "updating: python/algos/feature/ (stored 0%)\n",
      "updating: python/algos/feature/woe_encoder.py (deflated 64%)\n",
      "updating: python/algos/feature/.ipynb_checkpoints/ (stored 0%)\n",
      "updating: python/algos/feature/.ipynb_checkpoints/target_encoder-checkpoint.py (deflated 51%)\n",
      "updating: python/algos/feature/sequential_encoder.py (deflated 60%)\n",
      "updating: python/algos/feature/target_encoder.py (deflated 51%)\n",
      "updating: python/algos/feature/neg_sampler.py (deflated 66%)\n",
      "updating: python/algos/feature/__pycache__/ (stored 0%)\n",
      "updating: python/algos/feature/__pycache__/woe_encoder.cpython-38.pyc (deflated 43%)\n",
      "updating: python/algos/feature/__pycache__/target_encoder.cpython-38.pyc (deflated 36%)\n",
      "updating: python/algos/feature/__pycache__/__init__.cpython-38.pyc (deflated 29%)\n",
      "updating: python/algos/feature/__pycache__/sequential_encoder.cpython-38.pyc (deflated 53%)\n",
      "updating: python/algos/feature/__pycache__/neg_sampler.cpython-38.pyc (deflated 50%)\n",
      "updating: python/algos/feature/__init__.py (deflated 41%)\n",
      "updating: python/algos/dcn_v2_net.py (deflated 75%)\n",
      "updating: python/algos/autoint_net.py (deflated 70%)\n",
      "updating: python/algos/twotower/ (stored 0%)\n",
      "updating: python/algos/twotower/retrieval_metric.py (deflated 68%)\n",
      "updating: python/algos/twotower/simplex/ (stored 0%)\n",
      "updating: python/algos/twotower/simplex/simplex_agent.py (deflated 64%)\n",
      "updating: python/algos/twotower/simplex/simplex_net.py (deflated 71%)\n",
      "updating: python/algos/twotower/simplex/__init__.py (deflated 39%)\n",
      "updating: python/algos/twotower/two_tower_retrieval.py (deflated 81%)\n",
      "updating: python/algos/twotower/__init__.py (deflated 40%)\n",
      "updating: python/algos/twotower/dssm/ (stored 0%)\n",
      "updating: python/algos/twotower/dssm/dssm_agent.py (deflated 72%)\n",
      "updating: python/algos/twotower/dssm/dssm_net.py (deflated 78%)\n",
      "updating: python/algos/twotower/dssm/__init__.py (deflated 41%)\n",
      "updating: python/algos/pipeline/ (stored 0%)\n",
      "updating: python/algos/pipeline/deep_ctr.py (deflated 77%)\n",
      "updating: python/algos/pipeline/utils/ (stored 0%)\n",
      "updating: python/algos/pipeline/utils/class_utils.py (deflated 52%)\n",
      "updating: python/algos/pipeline/utils/dict_utils.py (deflated 39%)\n",
      "updating: python/algos/pipeline/utils/constants.py (deflated 42%)\n",
      "updating: python/algos/pipeline/utils/__pycache__/ (stored 0%)\n",
      "updating: python/algos/pipeline/utils/__pycache__/constants.cpython-38.pyc (deflated 30%)\n",
      "updating: python/algos/pipeline/utils/__pycache__/logger.cpython-38.pyc (deflated 30%)\n",
      "updating: python/algos/pipeline/utils/__pycache__/dict_utils.cpython-38.pyc (deflated 33%)\n",
      "updating: python/algos/pipeline/utils/__pycache__/__init__.cpython-38.pyc (deflated 23%)\n",
      "updating: python/algos/pipeline/utils/__pycache__/class_utils.cpython-38.pyc (deflated 37%)\n",
      "updating: python/algos/pipeline/utils/__init__.py (deflated 38%)\n",
      "updating: python/algos/pipeline/utils/logger.py (deflated 48%)\n",
      "updating: python/algos/pipeline/i2i_retrieval.py (deflated 75%)\n",
      "updating: python/algos/pipeline/data_loader.py (deflated 67%)\n",
      "updating: python/algos/pipeline/popular_retrieval.py (deflated 68%)\n",
      "updating: python/algos/pipeline/.ipynb_checkpoints/ (stored 0%)\n",
      "updating: python/algos/pipeline/.ipynb_checkpoints/deep_ctr-checkpoint.py (deflated 77%)\n",
      "updating: python/algos/pipeline/.ipynb_checkpoints/mongodb_dumper-checkpoint.py (deflated 62%)\n",
      "updating: python/algos/pipeline/.ipynb_checkpoints/init_spark-checkpoint.py (deflated 61%)\n",
      "updating: python/algos/pipeline/.ipynb_checkpoints/common_validators-checkpoint.py (deflated 66%)\n",
      "updating: python/algos/pipeline/.ipynb_checkpoints/i2i_retrieval-checkpoint.py (deflated 75%)\n",
      "updating: python/algos/pipeline/.ipynb_checkpoints/popular_retrieval-checkpoint.py (deflated 68%)\n",
      "updating: python/algos/pipeline/mongodb_dumper.py (deflated 62%)\n",
      "updating: python/algos/pipeline/common_validators.py (deflated 66%)\n",
      "updating: python/algos/pipeline/__pycache__/ (stored 0%)\n",
      "updating: python/algos/pipeline/__pycache__/i2i_retrieval.cpython-38.pyc (deflated 52%)\n",
      "updating: python/algos/pipeline/__pycache__/deep_ctr.cpython-38.pyc (deflated 53%)\n",
      "updating: python/algos/pipeline/__pycache__/popular_retrieval.cpython-38.pyc (deflated 46%)\n",
      "updating: python/algos/pipeline/__pycache__/mongodb_dumper.cpython-38.pyc (deflated 42%)\n",
      "updating: python/algos/pipeline/__pycache__/common_validators.cpython-38.pyc (deflated 55%)\n",
      "updating: python/algos/pipeline/__pycache__/data_loader.cpython-38.pyc (deflated 48%)\n",
      "updating: python/algos/pipeline/__pycache__/init_spark.cpython-38.pyc (deflated 41%)\n",
      "updating: python/algos/pipeline/__pycache__/__init__.cpython-38.pyc (deflated 43%)\n",
      "updating: python/algos/pipeline/__init__.py (deflated 52%)\n",
      "updating: python/algos/pipeline/init_spark.py (deflated 61%)\n",
      "updating: python/algos/tuner/ (stored 0%)\n",
      "updating: python/algos/tuner/base_tuner.py (deflated 70%)\n",
      "updating: python/algos/dcn_net.py (deflated 69%)\n",
      "updating: python/algos/pnn_net.py (deflated 70%)\n",
      "updating: python/algos/multitask/ (stored 0%)\n",
      "updating: python/algos/multitask/esmm/ (stored 0%)\n",
      "updating: python/algos/multitask/esmm/esmm_agent.py (deflated 74%)\n",
      "updating: python/algos/multitask/esmm/esmm_net.py (deflated 69%)\n",
      "updating: python/algos/multitask/__init__.py (deflated 43%)\n",
      "updating: python/algos/multitask/mmoe/ (stored 0%)\n",
      "updating: python/algos/multitask/mmoe/mmoe_net.py (deflated 75%)\n",
      "updating: python/algos/multitask/mmoe/mmoe_agent.py (deflated 70%)\n",
      "updating: python/algos/xdeepfm_net.py (deflated 71%)\n",
      "updating: python/algos/layers.py (deflated 79%)\n",
      "updating: python/algos/item_cf_retrieval.py (deflated 77%)\n",
      "updating: python/algos/fwfm_net.py (deflated 69%)\n",
      "updating: python/algos/ffm_net.py (deflated 71%)\n",
      "updating: python/algos/__pycache__/ (stored 0%)\n",
      "updating: python/algos/__pycache__/layers.cpython-38.pyc (deflated 59%)\n",
      "updating: python/algos/__pycache__/widedeep_net.cpython-38.pyc (deflated 41%)\n",
      "updating: python/algos/__pycache__/item_cf_retrieval.cpython-38.pyc (deflated 54%)\n",
      "updating: python/algos/__pycache__/__init__.cpython-38.pyc (deflated 21%)\n",
      "updating: python/algos/widedeep_net.py (deflated 68%)\n",
      "updating: python/algos/sequential/ (stored 0%)\n",
      "updating: python/algos/sequential/din/ (stored 0%)\n",
      "updating: python/algos/sequential/din/din_net.py (deflated 76%)\n",
      "updating: python/algos/sequential/dien/ (stored 0%)\n",
      "updating: python/algos/sequential/dien/dien_net.py (deflated 79%)\n",
      "updating: python/algos/sequential/dien/dien_agent.py (deflated 61%)\n",
      "updating: python/algos/sequential/__init__.py (deflated 41%)\n",
      "updating: python/algos/__init__.py (deflated 38%)\n",
      "updating: python/tests/ (stored 0%)\n",
      "updating: python/tests/mnist_mlp_eval.py (deflated 57%)\n",
      "updating: python/tests/mnist_mlp.py (deflated 59%)\n",
      "updating: python/tests/sparse_mlp_export_demo.py (deflated 64%)\n",
      "updating: python/tests/sparse_wdl_grpc_test.py (deflated 52%)\n",
      "updating: python/tests/sparse_two_tower_export_demo.py (deflated 73%)\n",
      "updating: python/tests/dense_xgboost.py (deflated 49%)\n",
      "updating: python/tests/verify_java_tensor_serde.py (deflated 42%)\n",
      "updating: python/tests/requirements.txt (deflated 24%)\n",
      "updating: python/tests/sparse_wdl_export_test.py (deflated 66%)\n",
      "updating: python/tests/sparse_wdl_export_demo.py (deflated 64%)\n",
      "updating: python/tests/dense_xgboost_grpc_test.py (deflated 44%)\n",
      "updating: python/tests/embedding_bag_export.py (deflated 61%)\n",
      "updating: python/tests/two_tower_retrieval_milvus.py (deflated 81%)\n",
      "updating: python/ps/ (stored 0%)\n",
      "updating: python/ps/job.py (deflated 81%)\n",
      "updating: python/ps/__init__.py (deflated 42%)\n",
      "updating: python/scripts/ (stored 0%)\n",
      "updating: python/scripts/preprocessing/ (stored 0%)\n",
      "updating: python/scripts/preprocessing/example_requirements.txt (stored 0%)\n",
      "updating: python/scripts/preprocessing/preprocessor_service.py (deflated 69%)\n",
      "updating: python/scripts/preprocessing/example_preprocessor.py (deflated 54%)\n",
      "updating: python/scripts/preprocessing/test_example_preprocessor.py (deflated 51%)\n",
      "updating: python/scripts/consul/ (stored 0%)\n",
      "updating: python/scripts/consul/Dockerfile (deflated 44%)\n",
      "updating: python/scripts/consul/consul_watch_load.py (deflated 72%)\n",
      "updating: python/scripts/consul/build.sh (deflated 30%)\n",
      "updating: python/scripts/consul/create_consul_watch.sh (deflated 17%)\n",
      "updating: python/metaspore/ (stored 0%)\n",
      "updating: python/metaspore/url_utils.py (deflated 51%)\n",
      "updating: python/metaspore/estimator.py (deflated 85%)\n",
      "updating: python/metaspore/model.py (deflated 81%)\n",
      "updating: python/metaspore/distributed_trainer.py (deflated 76%)\n",
      "updating: python/metaspore/ps_launcher.py (deflated 68%)\n",
      "updating: python/metaspore/experiment.py (deflated 76%)\n",
      "updating: python/metaspore/distributed_tensor.py (deflated 80%)\n",
      "updating: python/metaspore/schema_utils.py (deflated 70%)\n",
      "updating: python/metaspore/nn/ (stored 0%)\n",
      "updating: python/metaspore/nn/wide_and_deep.py (deflated 75%)\n",
      "updating: python/metaspore/nn/fm.py (deflated 47%)\n",
      "updating: python/metaspore/nn/normalization.py (deflated 61%)\n",
      "updating: python/metaspore/nn/deep_fm.py (deflated 70%)\n",
      "updating: python/metaspore/nn/__init__.py (deflated 41%)\n",
      "updating: python/metaspore/s3_utils.py (deflated 73%)\n",
      "updating: python/metaspore/file_utils.py (deflated 69%)\n",
      "updating: python/metaspore/embedding.py (deflated 81%)\n",
      "updating: python/metaspore/agent.py (deflated 77%)\n",
      "updating: python/metaspore/metric.py (deflated 78%)\n",
      "updating: python/metaspore/feature_group.py (deflated 58%)\n",
      "updating: python/metaspore/job_utils.py (deflated 47%)\n",
      "updating: python/metaspore/initializer.py (deflated 76%)\n",
      "updating: python/metaspore/demo.py (deflated 54%)\n",
      "updating: python/metaspore/two_tower_ranking.py (deflated 80%)\n",
      "updating: python/metaspore/shell_utils.py (deflated 65%)\n",
      "updating: python/metaspore/output.py (deflated 56%)\n",
      "updating: python/metaspore/network_utils.py (deflated 52%)\n",
      "updating: python/metaspore/loss_utils.py (deflated 40%)\n",
      "updating: python/metaspore/spark.py (deflated 71%)\n",
      "updating: python/metaspore/name_utils.py (deflated 37%)\n",
      "updating: python/metaspore/two_tower_retrieval.py (deflated 83%)\n",
      "updating: python/metaspore/updater.py (deflated 81%)\n",
      "updating: python/metaspore/patching_pickle.py (deflated 72%)\n",
      "updating: python/metaspore/cast.py (deflated 60%)\n",
      "updating: python/metaspore/stack_trace_utils.py (deflated 43%)\n",
      "updating: python/metaspore/__init__.py (deflated 68%)\n",
      "updating: python/metaspore/swing_retrieval.py (deflated 79%)\n",
      "updating: python/metaspore/compat/ (stored 0%)\n",
      "updating: python/metaspore/compat/ps/ (stored 0%)\n",
      "updating: python/metaspore/compat/ps/__init__.py (deflated 66%)\n",
      "updating: python/metaspore/compat/__init__.py (deflated 52%)\n",
      "updating: python/metaspore/input.py (deflated 59%)\n",
      "updating: python/algos/pipeline/utils/.ipynb_checkpoints/ (stored 0%)\n",
      "updating: python/algos/pipeline/utils/.ipynb_checkpoints/constants-checkpoint.py (deflated 42%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: An illegal reflective access operation has occurred\n",
      "WARNING: Illegal reflective access by org.apache.spark.unsafe.Platform (file:/usr/local/lib/python3.8/dist-packages/pyspark/jars/spark-unsafe_2.12-3.1.2.jar) to constructor java.nio.DirectByteBuffer(long,int)\n",
      "WARNING: Please consider reporting this to the maintainers of org.apache.spark.unsafe.Platform\n",
      "WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations\n",
      "WARNING: All illegal access operations will be denied in a future release\n",
      "22/10/13 18:25:14 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Debug -- spark init\n",
      "Debug -- version: 3.1.2\n",
      "Debug -- applicaitonId: spark-application-1665656716536\n",
      "Debug -- uiWebUrl: http://172.16.0.156:4040\n"
     ]
    }
   ],
   "source": [
    "# spark\n",
    "params = {}\n",
    "params['local'] = False\n",
    "params['app_name'] = 'Amazon Fashion Dataset'\n",
    "params['batch_size'] = 512\n",
    "params['worker_count'] = 4\n",
    "params['worker_cpu'] = 4\n",
    "params['server_count'] = 4\n",
    "params['server_cpu'] = 4\n",
    "params['worker_memory'] = '6G'\n",
    "params['server_memory'] = '6G'\n",
    "params['coordinator_memory'] = '2G'\n",
    "\n",
    "# input files\n",
    "product_path = '${MY_S3_BUCKET}/sass/ecommerce/amazonfashion/dataset/amazon_fashion_product.json'\n",
    "review_path = '${MY_S3_BUCKET}/sass/ecommerce/amazonfashion/dataset/amazon_fashion_review.json'\n",
    "item2image_path = '${MY_S3_BUCKET}/xwb/tmp/amazon_fashion_id2img.valid.tsv'\n",
    "category_valid_path = '${MY_S3_BUCKET}/xwb/tmp/amazon_fashion_category_valid.parquet'\n",
    "item_ids_aug_path = '${MY_S3_BUCKET}/xwb/tmp/item_ids.aug.tsv'\n",
    "\n",
    "# output files\n",
    "user_path = '${MY_S3_BUCKET}/sass/ecommerce/amazonfashion/dataset/amazon_fashion_user.parquet'\n",
    "item_path = '${MY_S3_BUCKET}/sass/ecommerce/amazonfashion/dataset/amazon_fashion_item.parquet'\n",
    "interaction_path = '${MY_S3_BUCKET}/sass/ecommerce/amazonfashion/dataset/amazon_fashion_interaction.parquet'\n",
    "\n",
    "spark = init_spark(**params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ed51fe7b-6f77-4fc4-845a-28ea2d765c9a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:29:48.086067Z",
     "iopub.status.busy": "2022-10-13T10:29:48.085536Z",
     "iopub.status.idle": "2022-10-13T10:30:01.273305Z",
     "shell.execute_reply": "2022-10-13T10:30:01.272539Z",
     "shell.execute_reply.started": "2022-10-13T10:29:48.086042Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 1:==================================================>      (22 + 3) / 25]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Debug -- item dataset count: 2685059\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "product_dataset = read_dataset(spark, product_path)\n",
    "product_dataset = product_dataset.distinct()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "66a35e18-afac-4b39-9980-75a931ab1ad2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:30:01.274424Z",
     "iopub.status.busy": "2022-10-13T10:30:01.274183Z",
     "iopub.status.idle": "2022-10-13T10:30:01.282732Z",
     "shell.execute_reply": "2022-10-13T10:30:01.282006Z",
     "shell.execute_reply.started": "2022-10-13T10:30:01.274403Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- brand: string (nullable = true)\n",
      " |-- category: array (nullable = true)\n",
      " |    |-- element: string (containsNull = true)\n",
      " |-- category_levels: long (nullable = true)\n",
      " |-- description: array (nullable = true)\n",
      " |    |-- element: string (containsNull = true)\n",
      " |-- image: string (nullable = true)\n",
      " |-- item_id: string (nullable = true)\n",
      " |-- price: string (nullable = true)\n",
      " |-- title: string (nullable = true)\n",
      " |-- url: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "product_dataset.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f82297a-e1ef-47cc-a30b-433f59eddadc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fcdfc05e-c774-4e64-b2b2-a93a3b8476d5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:30:01.284309Z",
     "iopub.status.busy": "2022-10-13T10:30:01.283688Z",
     "iopub.status.idle": "2022-10-13T10:30:03.321529Z",
     "shell.execute_reply": "2022-10-13T10:30:03.320795Z",
     "shell.execute_reply.started": "2022-10-13T10:30:01.284284Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- item_id: string (nullable = true)\n",
      " |-- image: string (nullable = true)\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 4:=======>                                                   (1 + 7) / 8]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+----------------------------------------------------------------+\n",
      "|item_id   |image                                                           |\n",
      "+----------+----------------------------------------------------------------+\n",
      "|7890000286|https://images-na.ssl-images-amazon.com/images/I/41ZTltTw8HL.jpg|\n",
      "|9830030296|https://images-na.ssl-images-amazon.com/images/I/413guZeLSjL.jpg|\n",
      "|B0000731JZ|https://images-na.ssl-images-amazon.com/images/I/31VOsl2X9JL.jpg|\n",
      "|B0000E09PI|https://images-na.ssl-images-amazon.com/images/I/515sEqEExkL.jpg|\n",
      "|B00023T290|https://images-na.ssl-images-amazon.com/images/I/41jPArV9aFL.jpg|\n",
      "|B00024WC7S|https://images-na.ssl-images-amazon.com/images/I/41vs4F7x-IL.jpg|\n",
      "|B00024WLAQ|https://images-na.ssl-images-amazon.com/images/I/31Hcek6lg0L.jpg|\n",
      "|B00024WOTY|https://images-na.ssl-images-amazon.com/images/I/41DX2nY6FfL.jpg|\n",
      "|B00027MV3A|https://images-na.ssl-images-amazon.com/images/I/41BJMYXA8RL.jpg|\n",
      "|B00028SNNQ|https://images-na.ssl-images-amazon.com/images/I/31ME4JX2HFL.jpg|\n",
      "|B0002PELQ2|https://images-na.ssl-images-amazon.com/images/I/51VlIk1wvoL.jpg|\n",
      "|B0002PYIXI|https://images-na.ssl-images-amazon.com/images/I/41knvVYIKNL.jpg|\n",
      "|B000632S9M|https://images-na.ssl-images-amazon.com/images/I/417a1Z6OFoL.jpg|\n",
      "|B00064UTM4|https://images-na.ssl-images-amazon.com/images/I/41aUWi1iT6L.jpg|\n",
      "|B000659XZC|https://images-na.ssl-images-amazon.com/images/I/41MhHWFwJgL.jpg|\n",
      "|B00065G1B6|https://images-na.ssl-images-amazon.com/images/I/41w4jWqmTNL.jpg|\n",
      "|B0006B12AK|https://images-na.ssl-images-amazon.com/images/I/11B-JdMFrwL.jpg|\n",
      "|B0006PLE9K|https://images-na.ssl-images-amazon.com/images/I/21RgmMgg1zL.jpg|\n",
      "|B0006U6BO8|https://images-na.ssl-images-amazon.com/images/I/41xDHbetX2L.jpg|\n",
      "|B0007QMU0K|https://images-na.ssl-images-amazon.com/images/I/3166WLWdBbL.jpg|\n",
      "+----------+----------------------------------------------------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "item2image_df = spark.read.option(\"delimiter\", \"\\t\").csv(item2image_path)\n",
    "item2image_df = item2image_df.withColumnRenamed('_c0', 'item_id')\n",
    "item2image_df = item2image_df.withColumnRenamed('_c1', 'image')\n",
    "item2image_df = item2image_df.distinct()\n",
    "item2image_df.printSchema()\n",
    "item2image_df.show(20, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "27917f9a-7c04-4b03-ad7a-b37f52df21c8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:30:03.322452Z",
     "iopub.status.busy": "2022-10-13T10:30:03.322243Z",
     "iopub.status.idle": "2022-10-13T10:30:04.564030Z",
     "shell.execute_reply": "2022-10-13T10:30:04.563332Z",
     "shell.execute_reply.started": "2022-10-13T10:30:03.322434Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- category: array (nullable = true)\n",
      " |    |-- element: string (containsNull = true)\n",
      " |-- category_path: string (nullable = true)\n",
      " |-- valid_level: long (nullable = true)\n",
      "\n",
      "+------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------+-----------+\n",
      "|category                                                                                        |category_path                                                                                 |valid_level|\n",
      "+------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------+-----------+\n",
      "|[Clothing, Shoes & Jewelry, Luggage & Travel Gear, Umbrellas, Folding Umbrellas]                |Clothing, Shoes & Jewelry->Luggage & Travel Gear->Umbrellas->Folding Umbrellas                |3          |\n",
      "|[Clothing, Shoes & Jewelry, Men, Accessories, Hats & Caps, Baseball Caps]                       |Clothing, Shoes & Jewelry->Men->Accessories->Hats & Caps->Baseball Caps                       |0          |\n",
      "|[Clothing, Shoes & Jewelry, Women, Clothing, Dresses, Casual]                                   |Clothing, Shoes & Jewelry->Women->Clothing->Dresses->Casual                                   |2          |\n",
      "|[Clothing, Shoes & Jewelry, Women, Clothing, Dresses, Wedding Dresses]                          |Clothing, Shoes & Jewelry->Women->Clothing->Dresses->Wedding Dresses                          |2          |\n",
      "|[Clothing, Shoes & Jewelry, Traditional & Cultural Wear, European]                              |Clothing, Shoes & Jewelry->Traditional & Cultural Wear->European                              |0          |\n",
      "|[Clothing, Shoes & Jewelry, Women, Clothing, Tops, Tees & Blouses, Blouses & Button-Down Shirts]|Clothing, Shoes & Jewelry->Women->Clothing->Tops, Tees & Blouses->Blouses & Button-Down Shirts|1          |\n",
      "|[Clothing, Shoes & Jewelry, Women, Clothing, Dresses, Cocktail]                                 |Clothing, Shoes & Jewelry->Women->Clothing->Dresses->Cocktail                                 |2          |\n",
      "|[Clothing, Shoes & Jewelry, Men, Clothing, Shorts, Cargo]                                       |Clothing, Shoes & Jewelry->Men->Clothing->Shorts->Cargo                                       |1          |\n",
      "|[Clothing, Shoes & Jewelry, Women, Clothing, Socks & Hosiery, Casual Socks]                     |Clothing, Shoes & Jewelry->Women->Clothing->Socks & Hosiery->Casual Socks                     |2          |\n",
      "|[Clothing, Shoes & Jewelry, Women, Jewelry, Necklaces, Chain Material: Lace]                    |Clothing, Shoes & Jewelry->Women->Jewelry->Necklaces->Chain Material: Lace                    |3          |\n",
      "|[Clothing, Shoes & Jewelry, Women, Shoes, Fashion Sneakers, canvas]                             |Clothing, Shoes & Jewelry->Women->Shoes->Fashion Sneakers->canvas                             |3          |\n",
      "|[Clothing, Shoes & Jewelry, Men, Jewelry, Rings, Colour: Black and Red]                         |Clothing, Shoes & Jewelry->Men->Jewelry->Rings->Colour: Black and Red                         |3          |\n",
      "|[Clothing, Shoes & Jewelry, Men, Accessories]                                                   |Clothing, Shoes & Jewelry->Men->Accessories                                                   |0          |\n",
      "|[Clothing, Shoes & Jewelry, Novelty & More, Clothing, Novelty, Men]                             |Clothing, Shoes & Jewelry->Novelty & More->Clothing->Novelty->Men                             |0          |\n",
      "|[Clothing, Shoes & Jewelry, Women, Handbags & Wallets, Shoulder Bags]                           |Clothing, Shoes & Jewelry->Women->Handbags & Wallets->Shoulder Bags                           |0          |\n",
      "|[Clothing, Shoes & Jewelry, Women, Clothing, Swimsuits & Cover Ups, Bikinis]                    |Clothing, Shoes & Jewelry->Women->Clothing->Swimsuits & Cover Ups->Bikinis                    |0          |\n",
      "|[Clothing, Shoes & Jewelry, Women, Shoes, Rubber, Slip on boat shoes style]                     |Clothing, Shoes & Jewelry->Women->Shoes->Rubber->Slip on boat shoes style                     |3          |\n",
      "|[Clothing, Shoes & Jewelry, Boys, Clothing, Tops & Tees, Tees]                                  |Clothing, Shoes & Jewelry->Boys->Clothing->Tops & Tees->Tees                                  |4          |\n",
      "|[Clothing, Shoes & Jewelry, Novelty & More, Clothing, Novelty, Girls]                           |Clothing, Shoes & Jewelry->Novelty & More->Clothing->Novelty->Girls                           |0          |\n",
      "|[Clothing, Shoes & Jewelry, Novelty & More, Clothing, Novelty, Women]                           |Clothing, Shoes & Jewelry->Novelty & More->Clothing->Novelty->Women                           |0          |\n",
      "+------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------+-----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "category_valid = spark.read.parquet(category_valid_path)\n",
    "category_valid.printSchema()\n",
    "category_valid.show(20, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "625f9edf-ffa0-4af2-acbe-e053cbb9a3cf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:30:04.565493Z",
     "iopub.status.busy": "2022-10-13T10:30:04.564952Z",
     "iopub.status.idle": "2022-10-13T10:30:41.353430Z",
     "shell.execute_reply": "2022-10-13T10:30:41.352747Z",
     "shell.execute_reply.started": "2022-10-13T10:30:04.565470Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 22:================================================>     (178 + 4) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1123917\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "product_dataset = product_dataset.drop('image')\n",
    "product_dataset = product_dataset.join(item2image_df, on='item_id', how='leftouter')\n",
    "\n",
    "product_dataset = product_dataset.filter(F.col(\"image\").isNotNull())\n",
    "product_dataset = product_dataset.filter(F.col(\"image\") != '')\n",
    "product_dataset = product_dataset.filter(F.col(\"title\") != '')\n",
    "product_dataset = product_dataset.filter(F.length(F.col(\"title\")) <= 300)\n",
    "product_dataset = product_dataset.filter(F.length(F.col(\"title\")) >= 5)\n",
    "\n",
    "product_dataset = product_dataset.withColumn('category_path', F.concat_ws('->', 'category'))\n",
    "\n",
    "product_dataset = category_valid.filter(F.col('valid_level') > 0).select('category_path').distinct()\\\n",
    "                    .join(product_dataset, on='category_path', how='inner').drop('category_path')\n",
    "\n",
    "product_dataset.limit(20).toPandas()\n",
    "print(product_dataset.count())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e1db17fd-1232-483a-af93-4865c61dbba2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:30:41.356357Z",
     "iopub.status.busy": "2022-10-13T10:30:41.355808Z",
     "iopub.status.idle": "2022-10-13T10:30:41.359187Z",
     "shell.execute_reply": "2022-10-13T10:30:41.358586Z",
     "shell.execute_reply.started": "2022-10-13T10:30:41.356332Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# product_dataset.filter(F.col('item_id') == 'B018448KC8').show(1,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ae6e07d-ff56-4eb5-9a38-ae45f9a10e89",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6bace451-07bc-4eb8-aeb4-0298e19b2210",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:30:41.360476Z",
     "iopub.status.busy": "2022-10-13T10:30:41.360039Z",
     "iopub.status.idle": "2022-10-13T10:30:56.266255Z",
     "shell.execute_reply": "2022-10-13T10:30:56.265592Z",
     "shell.execute_reply.started": "2022-10-13T10:30:41.360451Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 25:==============================================>         (19 + 4) / 23]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Debug -- item dataset count: 32292099\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "review_dataset = read_dataset(spark, review_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3aa529a2-6c8c-4072-9235-1d8880ea5543",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:30:56.267423Z",
     "iopub.status.busy": "2022-10-13T10:30:56.267105Z",
     "iopub.status.idle": "2022-10-13T10:30:56.284869Z",
     "shell.execute_reply": "2022-10-13T10:30:56.284087Z",
     "shell.execute_reply.started": "2022-10-13T10:30:56.267402Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pyspark.sql.functions as F\n",
    "product_subset = product_dataset.where(F.array_contains(F.col(\"category\"), 'Clothing') & \n",
    "                                       F.array_contains(F.col('category'), 'Women'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "11894078-93fe-43df-b14f-0d7fa0f0a97b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:30:56.286306Z",
     "iopub.status.busy": "2022-10-13T10:30:56.285839Z",
     "iopub.status.idle": "2022-10-13T10:31:11.147353Z",
     "shell.execute_reply": "2022-10-13T10:31:11.146670Z",
     "shell.execute_reply.started": "2022-10-13T10:30:56.286280Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>brand</th>\n",
       "      <th>category</th>\n",
       "      <th>category_levels</th>\n",
       "      <th>description</th>\n",
       "      <th>price</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>image</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>B0001GSYVA</td>\n",
       "      <td>FALKE</td>\n",
       "      <td>[Clothing, Shoes &amp; Jewelry, Women, Clothing, S...</td>\n",
       "      <td>5</td>\n",
       "      <td>[Incredibly soft sock made with skin-friendly ...</td>\n",
       "      <td>$15.75 - $24.00</td>\n",
       "      <td>Falke Women's Sensitive London Sock</td>\n",
       "      <td>https://www.amazon.com/dp/B0001GSYVA</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B000G0P2I4</td>\n",
       "      <td></td>\n",
       "      <td>[Clothing, Shoes &amp; Jewelry, Women, Clothing, J...</td>\n",
       "      <td>5</td>\n",
       "      <td>[Lee Relaxed Fit jeans are cut to fit your nat...</td>\n",
       "      <td></td>\n",
       "      <td>Lee Women's Relaxed Fit Straight Leg Jean</td>\n",
       "      <td>https://www.amazon.com/dp/B000G0P2I4</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>B000I38P74</td>\n",
       "      <td>Maison Jules</td>\n",
       "      <td>[Clothing, Shoes &amp; Jewelry, Women, Clothing, S...</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>$39.99</td>\n",
       "      <td>Maison Jules Women's Polka Dot Crew-Neck Pullo...</td>\n",
       "      <td>https://www.amazon.com/dp/B000I38P74</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>B0010ESQXS</td>\n",
       "      <td>Cashmere Boutique</td>\n",
       "      <td>[Clothing, Shoes &amp; Jewelry, Women, Clothing, C...</td>\n",
       "      <td>5</td>\n",
       "      <td>[Luxurious, soft and really warm, this pea coa...</td>\n",
       "      <td>$229.00</td>\n",
       "      <td>Women's Cashmere Pea Coat</td>\n",
       "      <td>https://www.amazon.com/dp/B0010ESQXS</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B0014GY2H6</td>\n",
       "      <td>Devon &amp; Jones</td>\n",
       "      <td>[Clothing, Shoes &amp; Jewelry, Women, Clothing, S...</td>\n",
       "      <td>5</td>\n",
       "      <td>[Refined, feminine, lightweight warmth in 86% ...</td>\n",
       "      <td>$17.49 - $28.44</td>\n",
       "      <td>Devon &amp; Jones Pink Everyday Cardigan Sweater D...</td>\n",
       "      <td>https://www.amazon.com/dp/B0014GY2H6</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>B0016SJP9W</td>\n",
       "      <td>Bigmansland</td>\n",
       "      <td>[Clothing, Shoes &amp; Jewelry, Women, Clothing, T...</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>$21.99</td>\n",
       "      <td>Bigmansland Ladies Short Sleeve Easy Care, Soi...</td>\n",
       "      <td>https://www.amazon.com/dp/B0016SJP9W</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>B001ANMXXI</td>\n",
       "      <td>Principle Business Enterprises</td>\n",
       "      <td>[Clothing, Shoes &amp; Jewelry, Women, Clothing, S...</td>\n",
       "      <td>5</td>\n",
       "      <td>[Terries Slip Resistant Socks not only provide...</td>\n",
       "      <td>$6.52</td>\n",
       "      <td>Terries Slip Resistant Socks XX-LARGE - DOUBLE...</td>\n",
       "      <td>https://www.amazon.com/dp/B001ANMXXI</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>B001B16M7M</td>\n",
       "      <td>Sugar Lips</td>\n",
       "      <td>[Clothing, Shoes &amp; Jewelry, Women, Clothing, T...</td>\n",
       "      <td>5</td>\n",
       "      <td>[The original SUGARLIPS ribbed seamless tank t...</td>\n",
       "      <td>$14.95</td>\n",
       "      <td>Sugarlips Women's Misses Original Seamless Rib...</td>\n",
       "      <td>https://www.amazon.com/dp/B001B16M7M</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>B001CL8JXQ</td>\n",
       "      <td></td>\n",
       "      <td>[Clothing, Shoes &amp; Jewelry, Women, Clothing, C...</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td></td>\n",
       "      <td>AK Anne Klein Women's Single Breasted Swing Coat</td>\n",
       "      <td>https://www.amazon.com/dp/B001CL8JXQ</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>B001JBZ1MQ</td>\n",
       "      <td>Harriton</td>\n",
       "      <td>[Clothing, Shoes &amp; Jewelry, Women, Clothing, T...</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>$19.04 - $25.81</td>\n",
       "      <td>Harriton Ladies' Long Sleeve Twill Shirt, White</td>\n",
       "      <td>https://www.amazon.com/dp/B001JBZ1MQ</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      item_id                           brand  \\\n",
       "0  B0001GSYVA                           FALKE   \n",
       "1  B000G0P2I4                                   \n",
       "2  B000I38P74                    Maison Jules   \n",
       "3  B0010ESQXS               Cashmere Boutique   \n",
       "4  B0014GY2H6                   Devon & Jones   \n",
       "5  B0016SJP9W                     Bigmansland   \n",
       "6  B001ANMXXI  Principle Business Enterprises   \n",
       "7  B001B16M7M                      Sugar Lips   \n",
       "8  B001CL8JXQ                                   \n",
       "9  B001JBZ1MQ                        Harriton   \n",
       "\n",
       "                                            category  category_levels  \\\n",
       "0  [Clothing, Shoes & Jewelry, Women, Clothing, S...                5   \n",
       "1  [Clothing, Shoes & Jewelry, Women, Clothing, J...                5   \n",
       "2  [Clothing, Shoes & Jewelry, Women, Clothing, S...                5   \n",
       "3  [Clothing, Shoes & Jewelry, Women, Clothing, C...                5   \n",
       "4  [Clothing, Shoes & Jewelry, Women, Clothing, S...                5   \n",
       "5  [Clothing, Shoes & Jewelry, Women, Clothing, T...                5   \n",
       "6  [Clothing, Shoes & Jewelry, Women, Clothing, S...                5   \n",
       "7  [Clothing, Shoes & Jewelry, Women, Clothing, T...                5   \n",
       "8  [Clothing, Shoes & Jewelry, Women, Clothing, C...                5   \n",
       "9  [Clothing, Shoes & Jewelry, Women, Clothing, T...                5   \n",
       "\n",
       "                                         description            price  \\\n",
       "0  [Incredibly soft sock made with skin-friendly ...  $15.75 - $24.00   \n",
       "1  [Lee Relaxed Fit jeans are cut to fit your nat...                    \n",
       "2                                               None           $39.99   \n",
       "3  [Luxurious, soft and really warm, this pea coa...          $229.00   \n",
       "4  [Refined, feminine, lightweight warmth in 86% ...  $17.49 - $28.44   \n",
       "5                                               None           $21.99   \n",
       "6  [Terries Slip Resistant Socks not only provide...            $6.52   \n",
       "7  [The original SUGARLIPS ribbed seamless tank t...           $14.95   \n",
       "8                                               None                    \n",
       "9                                               None  $19.04 - $25.81   \n",
       "\n",
       "                                               title  \\\n",
       "0                Falke Women's Sensitive London Sock   \n",
       "1          Lee Women's Relaxed Fit Straight Leg Jean   \n",
       "2  Maison Jules Women's Polka Dot Crew-Neck Pullo...   \n",
       "3                          Women's Cashmere Pea Coat   \n",
       "4  Devon & Jones Pink Everyday Cardigan Sweater D...   \n",
       "5  Bigmansland Ladies Short Sleeve Easy Care, Soi...   \n",
       "6  Terries Slip Resistant Socks XX-LARGE - DOUBLE...   \n",
       "7  Sugarlips Women's Misses Original Seamless Rib...   \n",
       "8   AK Anne Klein Women's Single Breasted Swing Coat   \n",
       "9    Harriton Ladies' Long Sleeve Twill Shirt, White   \n",
       "\n",
       "                                    url  \\\n",
       "0  https://www.amazon.com/dp/B0001GSYVA   \n",
       "1  https://www.amazon.com/dp/B000G0P2I4   \n",
       "2  https://www.amazon.com/dp/B000I38P74   \n",
       "3  https://www.amazon.com/dp/B0010ESQXS   \n",
       "4  https://www.amazon.com/dp/B0014GY2H6   \n",
       "5  https://www.amazon.com/dp/B0016SJP9W   \n",
       "6  https://www.amazon.com/dp/B001ANMXXI   \n",
       "7  https://www.amazon.com/dp/B001B16M7M   \n",
       "8  https://www.amazon.com/dp/B001CL8JXQ   \n",
       "9  https://www.amazon.com/dp/B001JBZ1MQ   \n",
       "\n",
       "                                               image  \n",
       "0  https://images-na.ssl-images-amazon.com/images...  \n",
       "1  https://images-na.ssl-images-amazon.com/images...  \n",
       "2  https://images-na.ssl-images-amazon.com/images...  \n",
       "3  https://images-na.ssl-images-amazon.com/images...  \n",
       "4  https://images-na.ssl-images-amazon.com/images...  \n",
       "5  https://images-na.ssl-images-amazon.com/images...  \n",
       "6  https://images-na.ssl-images-amazon.com/images...  \n",
       "7  https://images-na.ssl-images-amazon.com/images...  \n",
       "8  https://images-na.ssl-images-amazon.com/images...  \n",
       "9  https://images-na.ssl-images-amazon.com/images...  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "product_subset.limit(10).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3ff9dc4e-a215-4604-96ff-065d04423cff",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:31:11.148541Z",
     "iopub.status.busy": "2022-10-13T10:31:11.148203Z",
     "iopub.status.idle": "2022-10-13T10:31:11.915151Z",
     "shell.execute_reply": "2022-10-13T10:31:11.914549Z",
     "shell.execute_reply.started": "2022-10-13T10:31:11.148520Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>user_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>B002OB4234</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1408492800</td>\n",
       "      <td>A3CWY12FRVB32X</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B002OB4234</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1408492800</td>\n",
       "      <td>A2A5EQASDHE46A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>B002OB4234</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1407888000</td>\n",
       "      <td>AGPF7572AYJL7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>B002OB4234</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1407715200</td>\n",
       "      <td>A161SE7XSZ6I7E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B002OB4234</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1407715200</td>\n",
       "      <td>A1AG4H4O7U84KM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>B002OB4234</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1407715200</td>\n",
       "      <td>A255NTZA63ZYVU</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>B002OB4234</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1407715200</td>\n",
       "      <td>AN12CK1QJDO5V</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>B002OB4234</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1407628800</td>\n",
       "      <td>A116OI9U0XS14H</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>B002OF5AOU</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1371513600</td>\n",
       "      <td>A146IBKVP081TC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>B002OF5AOU</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1370649600</td>\n",
       "      <td>A36V6MNBY07A5C</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      item_id  rating   timestamp         user_id\n",
       "0  B002OB4234     5.0  1408492800  A3CWY12FRVB32X\n",
       "1  B002OB4234     5.0  1408492800  A2A5EQASDHE46A\n",
       "2  B002OB4234     5.0  1407888000   AGPF7572AYJL7\n",
       "3  B002OB4234     4.0  1407715200  A161SE7XSZ6I7E\n",
       "4  B002OB4234     4.0  1407715200  A1AG4H4O7U84KM\n",
       "5  B002OB4234     5.0  1407715200  A255NTZA63ZYVU\n",
       "6  B002OB4234     4.0  1407715200   AN12CK1QJDO5V\n",
       "7  B002OB4234     5.0  1407628800  A116OI9U0XS14H\n",
       "8  B002OF5AOU     5.0  1371513600  A146IBKVP081TC\n",
       "9  B002OF5AOU     3.0  1370649600  A36V6MNBY07A5C"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review_dataset.limit(10).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1023cf89-c11f-4448-abdc-68a8252ad2d4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:31:11.916401Z",
     "iopub.status.busy": "2022-10-13T10:31:11.915972Z",
     "iopub.status.idle": "2022-10-13T10:31:11.921338Z",
     "shell.execute_reply": "2022-10-13T10:31:11.920714Z",
     "shell.execute_reply.started": "2022-10-13T10:31:11.916381Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[item_id: string, rating: double, timestamp: bigint, user_id: string]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "62bb86e5-418c-41c4-b55d-ebbd6fd1dede",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:31:11.922514Z",
     "iopub.status.busy": "2022-10-13T10:31:11.922140Z",
     "iopub.status.idle": "2022-10-13T10:31:12.146973Z",
     "shell.execute_reply": "2022-10-13T10:31:12.146332Z",
     "shell.execute_reply.started": "2022-10-13T10:31:11.922495Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "review_dataset.registerTempTable('review_dataset') \n",
    "product_subset.registerTempTable('product_subset')\n",
    "query =\"\"\"\n",
    "select \n",
    "    ta.user_id\n",
    "from\n",
    "(\n",
    "    select\n",
    "        ta.user_id,\n",
    "        count(distinct ta.item_id, ta.timestamp) as review_count\n",
    "    from\n",
    "        review_dataset ta\n",
    "    join\n",
    "        product_subset tb\n",
    "    on \n",
    "        ta.item_id=tb.item_id\n",
    "    group by ta.user_id\n",
    ") ta\n",
    "where ta.review_count >= 20\n",
    "\"\"\"\n",
    "user_5core = spark.sql(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4be88ebd-814d-405a-8c93-3ba68c17489c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:31:12.148430Z",
     "iopub.status.busy": "2022-10-13T10:31:12.147881Z",
     "iopub.status.idle": "2022-10-13T10:31:46.766554Z",
     "shell.execute_reply": "2022-10-13T10:31:46.765874Z",
     "shell.execute_reply.started": "2022-10-13T10:31:12.148408Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1212"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_5core.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a16cefbb-7b73-414c-a985-beefd124c9c7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:31:46.767517Z",
     "iopub.status.busy": "2022-10-13T10:31:46.767291Z",
     "iopub.status.idle": "2022-10-13T10:32:28.314655Z",
     "shell.execute_reply": "2022-10-13T10:32:28.313791Z",
     "shell.execute_reply.started": "2022-10-13T10:31:46.767496Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AENSGPP7TSU4V</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A165P3MOJV3OVZ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A1N0KPRDV8M95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A8KBJDU553RF2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>A6H4H1E3OIIOT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>A3HWECWIGPTN67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>A159IR0ZIDS388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>A2LFM653MMEC9L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>A3EBPHKGHMBPPI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>A2IVKM8ZHFN7BE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>A1RXX1LVOQFH3I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>A384ZGXQCI2A2E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>A2DLJCAT4X048C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>A10MI1OIRQQ3L2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>A220STV60KRN7W</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>A2GXHN4EU46RM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>A2XTRTRAPHO821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>A3VG66KUM1WH4Z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>A2SXQ8QP7LORDO</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           user_id\n",
       "0    AENSGPP7TSU4V\n",
       "1   A3KRRVHYFDDU59\n",
       "2   A165P3MOJV3OVZ\n",
       "3    A1N0KPRDV8M95\n",
       "4    A8KBJDU553RF2\n",
       "5    A6H4H1E3OIIOT\n",
       "6   A3HWECWIGPTN67\n",
       "7   A159IR0ZIDS388\n",
       "8   A2LFM653MMEC9L\n",
       "9   A3EBPHKGHMBPPI\n",
       "10  A2IVKM8ZHFN7BE\n",
       "11  A1RXX1LVOQFH3I\n",
       "12  A384ZGXQCI2A2E\n",
       "13  A2DLJCAT4X048C\n",
       "14  A10MI1OIRQQ3L2\n",
       "15  A220STV60KRN7W\n",
       "16   A2GXHN4EU46RM\n",
       "17  A2XTRTRAPHO821\n",
       "18  A3VG66KUM1WH4Z\n",
       "19  A2SXQ8QP7LORDO"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_5core.limit(20).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2f4f5d61-4ace-4f2b-96c9-a2807aaf0740",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:32:28.316051Z",
     "iopub.status.busy": "2022-10-13T10:32:28.315683Z",
     "iopub.status.idle": "2022-10-13T10:32:28.340859Z",
     "shell.execute_reply": "2022-10-13T10:32:28.340139Z",
     "shell.execute_reply.started": "2022-10-13T10:32:28.316028Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "user_5core.registerTempTable('user_5core')\n",
    "\n",
    "query =\"\"\"\n",
    "select distinct\n",
    "    tb.*\n",
    "from\n",
    "    user_5core ta\n",
    "join\n",
    "    review_dataset tb\n",
    "on ta.user_id=tb.user_id\n",
    "\"\"\"\n",
    "review_5core = spark.sql(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "eefb0272-3f96-4b6f-a1b2-11ad9ded4e9a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:32:28.342340Z",
     "iopub.status.busy": "2022-10-13T10:32:28.341765Z",
     "iopub.status.idle": "2022-10-13T10:33:25.800800Z",
     "shell.execute_reply": "2022-10-13T10:33:25.800153Z",
     "shell.execute_reply.started": "2022-10-13T10:32:28.342318Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "105155"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review_5core.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9646b68d-094f-4096-abfe-79b1729c418f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:33:25.801943Z",
     "iopub.status.busy": "2022-10-13T10:33:25.801677Z",
     "iopub.status.idle": "2022-10-13T10:34:39.950086Z",
     "shell.execute_reply": "2022-10-13T10:34:39.949463Z",
     "shell.execute_reply.started": "2022-10-13T10:33:25.801922Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>user_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>B009MBQYVW</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1435190400</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B009Z1S6WY</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1521417600</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>B00ATUWZPO</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1444003200</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>B00BCRKVNQ</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1444003200</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B00BGVJUYY</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1454457600</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>B00BMTT40U</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1473897600</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>B00BNT7D20</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1444003200</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>B00BU96BJO</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1439942400</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>B00C01MI3E</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1436400000</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>B01CGV1FTK</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1527552000</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>B01D1R5Q3O</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1521417600</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>B01DBJQB18</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1521417600</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>B01E9GP8RK</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1473897600</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>B01FJ8OJK4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1510704000</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>B01FY9XN00</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1527552000</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>B01B8C277W</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1473033600</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>B01COCK8YO</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1472428800</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>B006Z8FYVG</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1454457600</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>B0071QVZI2</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1421884800</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>B0071SYHIA</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1444003200</td>\n",
       "      <td>A3KRRVHYFDDU59</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       item_id  rating   timestamp         user_id\n",
       "0   B009MBQYVW     3.0  1435190400  A3KRRVHYFDDU59\n",
       "1   B009Z1S6WY     5.0  1521417600  A3KRRVHYFDDU59\n",
       "2   B00ATUWZPO     3.0  1444003200  A3KRRVHYFDDU59\n",
       "3   B00BCRKVNQ     5.0  1444003200  A3KRRVHYFDDU59\n",
       "4   B00BGVJUYY     5.0  1454457600  A3KRRVHYFDDU59\n",
       "5   B00BMTT40U     5.0  1473897600  A3KRRVHYFDDU59\n",
       "6   B00BNT7D20     4.0  1444003200  A3KRRVHYFDDU59\n",
       "7   B00BU96BJO     5.0  1439942400  A3KRRVHYFDDU59\n",
       "8   B00C01MI3E     4.0  1436400000  A3KRRVHYFDDU59\n",
       "9   B01CGV1FTK     5.0  1527552000  A3KRRVHYFDDU59\n",
       "10  B01D1R5Q3O     4.0  1521417600  A3KRRVHYFDDU59\n",
       "11  B01DBJQB18     5.0  1521417600  A3KRRVHYFDDU59\n",
       "12  B01E9GP8RK     5.0  1473897600  A3KRRVHYFDDU59\n",
       "13  B01FJ8OJK4     5.0  1510704000  A3KRRVHYFDDU59\n",
       "14  B01FY9XN00     5.0  1527552000  A3KRRVHYFDDU59\n",
       "15  B01B8C277W     3.0  1473033600  A3KRRVHYFDDU59\n",
       "16  B01COCK8YO     2.0  1472428800  A3KRRVHYFDDU59\n",
       "17  B006Z8FYVG     1.0  1454457600  A3KRRVHYFDDU59\n",
       "18  B0071QVZI2     5.0  1421884800  A3KRRVHYFDDU59\n",
       "19  B0071SYHIA     4.0  1444003200  A3KRRVHYFDDU59"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "review_5core.limit(20).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "de711913-2924-4ccb-a440-5976b60abc1a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:34:39.951459Z",
     "iopub.status.busy": "2022-10-13T10:34:39.951076Z",
     "iopub.status.idle": "2022-10-13T10:34:39.970248Z",
     "shell.execute_reply": "2022-10-13T10:34:39.969521Z",
     "shell.execute_reply.started": "2022-10-13T10:34:39.951439Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "review_5core.registerTempTable('review_5core')\n",
    "\n",
    "query =\"\"\"\n",
    "select distinct\n",
    "    tb.*\n",
    "from\n",
    "    review_5core ta\n",
    "join\n",
    "    product_subset tb\n",
    "on ta.item_id=tb.item_id\n",
    "\"\"\"\n",
    "product_5core = spark.sql(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "576e26af-f489-45df-8e5b-83ceec244ea9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:34:39.971797Z",
     "iopub.status.busy": "2022-10-13T10:34:39.971249Z",
     "iopub.status.idle": "2022-10-13T10:35:28.551406Z",
     "shell.execute_reply": "2022-10-13T10:35:28.550653Z",
     "shell.execute_reply.started": "2022-10-13T10:34:39.971762Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>brand</th>\n",
       "      <th>category</th>\n",
       "      <th>category_levels</th>\n",
       "      <th>description</th>\n",
       "      <th>price</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>image</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>B0069J1Y2Y</td>\n",
       "      <td></td>\n",
       "      <td>[Clothing, Shoes &amp; Jewelry, Women, Clothing, T...</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td>$21.59 - $26.99</td>\n",
       "      <td>PattyBoutik Women's Off Shoulder Long Sleeve Top</td>\n",
       "      <td>https://www.amazon.com/dp/B0069J1Y2Y</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B006MQR4QY</td>\n",
       "      <td></td>\n",
       "      <td>[Clothing, Shoes &amp; Jewelry, Women, Clothing, C...</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td></td>\n",
       "      <td>AK Anne Klein Women's Ruffle Trench Coat</td>\n",
       "      <td>https://www.amazon.com/dp/B006MQR4QY</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>B006UCPQAQ</td>\n",
       "      <td></td>\n",
       "      <td>[Clothing, Shoes &amp; Jewelry, Women, Clothing, D...</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td></td>\n",
       "      <td>HE02713BK10, Black, 8US, Ever Pretty Chic Lace...</td>\n",
       "      <td>https://www.amazon.com/dp/B006UCPQAQ</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>B0078LJ5KU</td>\n",
       "      <td></td>\n",
       "      <td>[Clothing, Shoes &amp; Jewelry, Women, Clothing, T...</td>\n",
       "      <td>5</td>\n",
       "      <td>None</td>\n",
       "      <td></td>\n",
       "      <td>XOXO Juniors Smock Bottom Embellished Printed ...</td>\n",
       "      <td>https://www.amazon.com/dp/B0078LJ5KU</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B007P5S2JY</td>\n",
       "      <td></td>\n",
       "      <td>[Clothing, Shoes &amp; Jewelry, Women, Clothing, P...</td>\n",
       "      <td>5</td>\n",
       "      <td>[Self fabric waistband with smooth interior el...</td>\n",
       "      <td></td>\n",
       "      <td>PUMA Women's Velour Pant</td>\n",
       "      <td>https://www.amazon.com/dp/B007P5S2JY</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      item_id brand                                           category  \\\n",
       "0  B0069J1Y2Y        [Clothing, Shoes & Jewelry, Women, Clothing, T...   \n",
       "1  B006MQR4QY        [Clothing, Shoes & Jewelry, Women, Clothing, C...   \n",
       "2  B006UCPQAQ        [Clothing, Shoes & Jewelry, Women, Clothing, D...   \n",
       "3  B0078LJ5KU        [Clothing, Shoes & Jewelry, Women, Clothing, T...   \n",
       "4  B007P5S2JY        [Clothing, Shoes & Jewelry, Women, Clothing, P...   \n",
       "\n",
       "   category_levels                                        description  \\\n",
       "0                5                                               None   \n",
       "1                5                                               None   \n",
       "2                5                                               None   \n",
       "3                5                                               None   \n",
       "4                5  [Self fabric waistband with smooth interior el...   \n",
       "\n",
       "             price                                              title  \\\n",
       "0  $21.59 - $26.99   PattyBoutik Women's Off Shoulder Long Sleeve Top   \n",
       "1                            AK Anne Klein Women's Ruffle Trench Coat   \n",
       "2                   HE02713BK10, Black, 8US, Ever Pretty Chic Lace...   \n",
       "3                   XOXO Juniors Smock Bottom Embellished Printed ...   \n",
       "4                                            PUMA Women's Velour Pant   \n",
       "\n",
       "                                    url  \\\n",
       "0  https://www.amazon.com/dp/B0069J1Y2Y   \n",
       "1  https://www.amazon.com/dp/B006MQR4QY   \n",
       "2  https://www.amazon.com/dp/B006UCPQAQ   \n",
       "3  https://www.amazon.com/dp/B0078LJ5KU   \n",
       "4  https://www.amazon.com/dp/B007P5S2JY   \n",
       "\n",
       "                                               image  \n",
       "0  https://images-na.ssl-images-amazon.com/images...  \n",
       "1  https://images-na.ssl-images-amazon.com/images...  \n",
       "2  https://images-na.ssl-images-amazon.com/images...  \n",
       "3  https://images-na.ssl-images-amazon.com/images...  \n",
       "4  https://images-na.ssl-images-amazon.com/images...  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "product_5core.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "ea7b97e2-f61c-4d5d-a964-17b156f8a557",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:35:28.552689Z",
     "iopub.status.busy": "2022-10-13T10:35:28.552375Z",
     "iopub.status.idle": "2022-10-13T10:36:20.361371Z",
     "shell.execute_reply": "2022-10-13T10:36:20.360634Z",
     "shell.execute_reply.started": "2022-10-13T10:35:28.552658Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "16883"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "product_5core.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7f26c84-a70c-458b-a75b-feb6b3257bce",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "da5a81ba-d524-4a81-b42f-b67063a42024",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:36:20.362920Z",
     "iopub.status.busy": "2022-10-13T10:36:20.362331Z",
     "iopub.status.idle": "2022-10-13T10:36:20.367756Z",
     "shell.execute_reply": "2022-10-13T10:36:20.367171Z",
     "shell.execute_reply.started": "2022-10-13T10:36:20.362898Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nimport mimetypes, urllib\\nfrom pyspark.sql.types import BooleanType\\nfrom pyspark.sql.functions import udf\\n\\ndef is_url_image(url):    \\n    mimetype,encoding = mimetypes.guess_type(url)\\n    return (mimetype and mimetype.startswith(\\'image\\'))\\n\\ndef check_url(url):\\n    \"\"\"Returns True if the url returns a response code between 200-300,\\n       otherwise return False.\\n    \"\"\"\\n    try:\\n        headers = {\\n            \"Range\": \"bytes=0-10\",\\n            \"User-Agent\": \"MyTestAgent\",\\n            \"Accept\": \"*/*\"\\n        }\\n\\n        req = urllib.request.Request(url, headers=headers)\\n        response = urllib.request.urlopen(req)\\n        print(\\'response.code: \\', response.code)\\n        return response.code in range(200, 209)\\n    except Exception as e:\\n        print(\\'exception: \\', e)\\n        return False\\n\\nis_image_and_ready_udf = udf(lambda image: is_url_image(image) and check_url(image), BooleanType())\\n\\n\\nreview_5core = review_5core.limit(10000)\\n\\nproduct_5core = review_5core.select(\\'item_id\\').distinct().join(product_5core, on=\\'item_id\\', how=\\'inner\\')\\nproduct_5core = product_5core.filter(is_image_and_ready_udf(\\'image\\'))\\nproduct_5core.cache()\\n\\nreview_5core = product_5core.select(\\'item_id\\').distinct().join(review_5core, on=\\'item_id\\', how=\\'inner\\')\\nreview_5core.cache()\\n\\nuser_5core = review_5core.select(\\'user_id\\').distinct().join(user_5core, on=\\'user_id\\', how=\\'inner\\')\\nuser_5core.cache()\\n'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "import mimetypes, urllib\n",
    "from pyspark.sql.types import BooleanType\n",
    "from pyspark.sql.functions import udf\n",
    "\n",
    "def is_url_image(url):    \n",
    "    mimetype,encoding = mimetypes.guess_type(url)\n",
    "    return (mimetype and mimetype.startswith('image'))\n",
    "\n",
    "def check_url(url):\n",
    "    \"\"\"Returns True if the url returns a response code between 200-300,\n",
    "       otherwise return False.\n",
    "    \"\"\"\n",
    "    try:\n",
    "        headers = {\n",
    "            \"Range\": \"bytes=0-10\",\n",
    "            \"User-Agent\": \"MyTestAgent\",\n",
    "            \"Accept\": \"*/*\"\n",
    "        }\n",
    "\n",
    "        req = urllib.request.Request(url, headers=headers)\n",
    "        response = urllib.request.urlopen(req)\n",
    "        print('response.code: ', response.code)\n",
    "        return response.code in range(200, 209)\n",
    "    except Exception as e:\n",
    "        print('exception: ', e)\n",
    "        return False\n",
    "\n",
    "is_image_and_ready_udf = udf(lambda image: is_url_image(image) and check_url(image), BooleanType())\n",
    "\n",
    "\n",
    "review_5core = review_5core.limit(10000)\n",
    "\n",
    "product_5core = review_5core.select('item_id').distinct().join(product_5core, on='item_id', how='inner')\n",
    "product_5core = product_5core.filter(is_image_and_ready_udf('image'))\n",
    "product_5core.cache()\n",
    "\n",
    "review_5core = product_5core.select('item_id').distinct().join(review_5core, on='item_id', how='inner')\n",
    "review_5core.cache()\n",
    "\n",
    "user_5core = review_5core.select('user_id').distinct().join(user_5core, on='user_id', how='inner')\n",
    "user_5core.cache()\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a885226e-d2ec-4e55-a6e0-c4cc51f309b1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:36:20.368771Z",
     "iopub.status.busy": "2022-10-13T10:36:20.368560Z",
     "iopub.status.idle": "2022-10-13T10:36:21.020490Z",
     "shell.execute_reply": "2022-10-13T10:36:21.019829Z",
     "shell.execute_reply.started": "2022-10-13T10:36:20.368748Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- item_id: string (nullable = true)\n",
      " |-- extra_item_id: string (nullable = true)\n",
      "\n",
      "+----------+-------------+\n",
      "|item_id   |extra_item_id|\n",
      "+----------+-------------+\n",
      "|B0132JUHV2|B01DNUD5I2   |\n",
      "|B015ISU4KO|B01E4WUADQ   |\n",
      "|B016A5VW4Q|B01FNUPOF2   |\n",
      "|B016Q0076O|B013IUBXFE   |\n",
      "|B016ZDQEWI|B00VS4B1VY   |\n",
      "|B018JABMHW|B00R4TJJOS   |\n",
      "|B019IBMJKU|B01A4CDCLC   |\n",
      "|B01B4M40D0|B00LO4N7LK   |\n",
      "|B009FZA25Y|B00W3HY108   |\n",
      "|B00MN7REHA|B0194ZDORM   |\n",
      "|B00OB8UTQW|B01A9RP7WY   |\n",
      "|B00PV6UUVM|B01E7QKE5S   |\n",
      "|B00VYG4PZK|B013W7M3XE   |\n",
      "|B00XJQ4QAW|B00YV59XKI   |\n",
      "|B013GAHVVG|B00JGVAR34   |\n",
      "|B01B29G3V2|B01GU62EJM   |\n",
      "|B01CSXQ82A|B01FQWMFKO   |\n",
      "|B01CZ2F7QM|B00LO4N7LK   |\n",
      "|B00JDVH7QW|B01GGR0ICU   |\n",
      "|B00K3B6SX4|B018QC4HHU   |\n",
      "+----------+-------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "item_ids_aug_df = spark.read.option(\"delimiter\", \"\\t\").csv(item_ids_aug_path)\n",
    "item_ids_aug_df = item_ids_aug_df.withColumnRenamed('_c0', 'item_id')\n",
    "item_ids_aug_df = item_ids_aug_df.withColumnRenamed('_c1', 'extra_item_id')\n",
    "item_ids_aug_df = item_ids_aug_df.distinct()\n",
    "item_ids_aug_df.printSchema()\n",
    "item_ids_aug_df.show(20, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "53222810-8e2a-48be-b9b5-6f8cc09f5e3d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:36:21.022006Z",
     "iopub.status.busy": "2022-10-13T10:36:21.021390Z",
     "iopub.status.idle": "2022-10-13T10:36:21.026593Z",
     "shell.execute_reply": "2022-10-13T10:36:21.026003Z",
     "shell.execute_reply.started": "2022-10-13T10:36:21.021980Z"
    }
   },
   "outputs": [],
   "source": [
    "# review_5core = review_5core.orderBy(F.rand()).limit(10000)\n",
    "review_5core = review_5core.limit(100000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e80d2386-d821-48db-95c1-beba2c151dae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:36:21.030237Z",
     "iopub.status.busy": "2022-10-13T10:36:21.029799Z",
     "iopub.status.idle": "2022-10-13T10:37:24.545566Z",
     "shell.execute_reply": "2022-10-13T10:37:24.544694Z",
     "shell.execute_reply.started": "2022-10-13T10:36:21.030216Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 121:====================================================>(199 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "extra_review count:  52827\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "from time import time \n",
    "\n",
    "extra_review = review_5core.join(item_ids_aug_df, on='item_id', how='inner')\\\n",
    "                .drop('item_id', 'timestamp')\\\n",
    "                .withColumnRenamed('extra_item_id', 'item_id')\\\n",
    "                .withColumn('timestamp', F.lit(int(time())))\\\n",
    "                .select(*review_5core.columns)\n",
    "print('extra_review count: ', extra_review.count())\n",
    "\n",
    "review_5core = review_5core.union(extra_review)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a88899b0-202f-43d8-97ed-4b5024df4bf0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:37:24.546758Z",
     "iopub.status.busy": "2022-10-13T10:37:24.546456Z",
     "iopub.status.idle": "2022-10-13T10:37:25.328574Z",
     "shell.execute_reply": "2022-10-13T10:37:25.327882Z",
     "shell.execute_reply.started": "2022-10-13T10:37:24.546736Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[item_id: string, rating: double, timestamp: bigint, user_id: string]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_5core = review_5core.select('user_id').distinct().join(user_5core, on='user_id', how='inner')\n",
    "user_5core.cache()\n",
    "\n",
    "product_5core = review_5core.select('item_id').distinct().join(product_5core, on='item_id', how='inner')\n",
    "product_5core.cache()\n",
    "\n",
    "review_5core = review_5core.join(product_5core.select('item_id').distinct(), on='item_id', how='inner').select(*review_5core.columns)\n",
    "review_5core.cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "e91b3e44-0ff0-43a5-8711-f5a64ba02ade",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:37:25.330081Z",
     "iopub.status.busy": "2022-10-13T10:37:25.329492Z",
     "iopub.status.idle": "2022-10-13T10:37:25.354634Z",
     "shell.execute_reply": "2022-10-13T10:37:25.353958Z",
     "shell.execute_reply.started": "2022-10-13T10:37:25.330060Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from pyspark.sql.types import ArrayType\n",
    "\n",
    "def concat_list_field(dataset, array_join_sep=u'\\u0001'):\n",
    "    array_cols = [f.name for f in dataset.schema.fields if isinstance(f.dataType, ArrayType)]\n",
    "    for array_col in array_cols:\n",
    "        dataset = dataset.withColumn(array_col, F.concat_ws(array_join_sep, F.col(array_col)))\n",
    "    return dataset\n",
    "\n",
    "review_5core = concat_list_field(review_5core)\n",
    "user_5core = concat_list_field(user_5core)\n",
    "product_5core = concat_list_field(product_5core)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "23d8fb5c-a628-4852-97dd-30be15313658",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:37:25.356237Z",
     "iopub.status.busy": "2022-10-13T10:37:25.355603Z",
     "iopub.status.idle": "2022-10-13T10:40:20.500835Z",
     "shell.execute_reply": "2022-10-13T10:40:20.500101Z",
     "shell.execute_reply.started": "2022-10-13T10:37:25.356210Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                ]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "review_5core:  84611\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "user_5core:  1210\n",
      "product_5core:  16283\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "print('review_5core: ', review_5core.count())\n",
    "print('user_5core: ', user_5core.count())\n",
    "print('product_5core: ', product_5core.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8defc496-e041-4d5f-bcf3-d5f623a40d9a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "2e7c71aa-1577-4aec-b68c-3af04b0b2858",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:40:20.502377Z",
     "iopub.status.busy": "2022-10-13T10:40:20.501780Z",
     "iopub.status.idle": "2022-10-13T10:40:20.505246Z",
     "shell.execute_reply": "2022-10-13T10:40:20.504687Z",
     "shell.execute_reply.started": "2022-10-13T10:40:20.502354Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "user_path = '${MY_S3_BUCKET}/sass/ecommerce/amazonfashion/dataset/amazon_fashion_user.100K.extra.parquet'\n",
    "item_path = '${MY_S3_BUCKET}/sass/ecommerce/amazonfashion/dataset/amazon_fashion_item.100K.extra.parquet'\n",
    "interaction_path = '${MY_S3_BUCKET}/sass/ecommerce/amazonfashion/dataset/amazon_fashion_interaction.100K.extra.parquet'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "de950e59-8a93-4005-b461-376b05cb01a3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:40:20.506484Z",
     "iopub.status.busy": "2022-10-13T10:40:20.506014Z",
     "iopub.status.idle": "2022-10-13T10:40:21.600078Z",
     "shell.execute_reply": "2022-10-13T10:40:21.599478Z",
     "shell.execute_reply.started": "2022-10-13T10:40:20.506463Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>brand</th>\n",
       "      <th>category</th>\n",
       "      <th>category_levels</th>\n",
       "      <th>description</th>\n",
       "      <th>price</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>image</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>B0069J1Y2Y</td>\n",
       "      <td></td>\n",
       "      <td>Clothing, Shoes &amp; Jewelry\u0001Women\u0001Clothing\u0001Tops,...</td>\n",
       "      <td>5</td>\n",
       "      <td></td>\n",
       "      <td>$21.59 - $26.99</td>\n",
       "      <td>PattyBoutik Women's Off Shoulder Long Sleeve Top</td>\n",
       "      <td>https://www.amazon.com/dp/B0069J1Y2Y</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B006MQR4QY</td>\n",
       "      <td></td>\n",
       "      <td>Clothing, Shoes &amp; Jewelry\u0001Women\u0001Clothing\u0001Coats...</td>\n",
       "      <td>5</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>AK Anne Klein Women's Ruffle Trench Coat</td>\n",
       "      <td>https://www.amazon.com/dp/B006MQR4QY</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>B0078LJ5KU</td>\n",
       "      <td></td>\n",
       "      <td>Clothing, Shoes &amp; Jewelry\u0001Women\u0001Clothing\u0001Tops,...</td>\n",
       "      <td>5</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>XOXO Juniors Smock Bottom Embellished Printed ...</td>\n",
       "      <td>https://www.amazon.com/dp/B0078LJ5KU</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>B007P5S2JY</td>\n",
       "      <td></td>\n",
       "      <td>Clothing, Shoes &amp; Jewelry\u0001Women\u0001Clothing\u0001Pants...</td>\n",
       "      <td>5</td>\n",
       "      <td>Self fabric waistband with smooth interior ela...</td>\n",
       "      <td></td>\n",
       "      <td>PUMA Women's Velour Pant</td>\n",
       "      <td>https://www.amazon.com/dp/B007P5S2JY</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B007TUODIE</td>\n",
       "      <td></td>\n",
       "      <td>Clothing, Shoes &amp; Jewelry\u0001Women\u0001Clothing\u0001Dress...</td>\n",
       "      <td>5</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>MYNE Women's Gemma Dress</td>\n",
       "      <td>https://www.amazon.com/dp/B007TUODIE</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>B008EYPQFI</td>\n",
       "      <td></td>\n",
       "      <td>Clothing, Shoes &amp; Jewelry\u0001Women\u0001Clothing\u0001Jeans</td>\n",
       "      <td>4</td>\n",
       "      <td>Look one size smaller in this Not Your Daughte...</td>\n",
       "      <td>$37.81 - $158.00</td>\n",
       "      <td>NYDJ Women's Petite Size Alina Legging Jean</td>\n",
       "      <td>https://www.amazon.com/dp/B008EYPQFI</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>B008VO5UES</td>\n",
       "      <td>D.E.P.T.</td>\n",
       "      <td>Clothing, Shoes &amp; Jewelry\u0001Women\u0001Clothing\u0001Coats...</td>\n",
       "      <td>5</td>\n",
       "      <td>The classic military inspired coat is updated ...</td>\n",
       "      <td></td>\n",
       "      <td>D.E.P.T. Women's Fabulous Coat</td>\n",
       "      <td>https://www.amazon.com/dp/B008VO5UES</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>B009O90116</td>\n",
       "      <td></td>\n",
       "      <td>Clothing, Shoes &amp; Jewelry\u0001Women\u0001Clothing\u0001Tops,...</td>\n",
       "      <td>5</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Southpole Juniors' Lightweight Lace-Fabric Hen...</td>\n",
       "      <td>https://www.amazon.com/dp/B009O90116</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>B00A85Z7MI</td>\n",
       "      <td>G2 Chic</td>\n",
       "      <td>Clothing, Shoes &amp; Jewelry\u0001Women\u0001Clothing\u0001Tops,...</td>\n",
       "      <td>5</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>G2 Chic Women's Crochet Accent Cape Loose Top</td>\n",
       "      <td>https://www.amazon.com/dp/B00A85Z7MI</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>B00AEVHJZY</td>\n",
       "      <td>G2 Chic</td>\n",
       "      <td>Clothing, Shoes &amp; Jewelry\u0001Women\u0001Clothing\u0001Tops,...</td>\n",
       "      <td>5</td>\n",
       "      <td>Round neck, oversized, quarter-sleeves, bumpy ...</td>\n",
       "      <td></td>\n",
       "      <td>G2 Chic Women's Lightweight Solid Patterned La...</td>\n",
       "      <td>https://www.amazon.com/dp/B00AEVHJZY</td>\n",
       "      <td>https://images-na.ssl-images-amazon.com/images...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      item_id     brand                                           category  \\\n",
       "0  B0069J1Y2Y            Clothing, Shoes & Jewelry\u0001Women\u0001Clothing\u0001Tops,...   \n",
       "1  B006MQR4QY            Clothing, Shoes & Jewelry\u0001Women\u0001Clothing\u0001Coats...   \n",
       "2  B0078LJ5KU            Clothing, Shoes & Jewelry\u0001Women\u0001Clothing\u0001Tops,...   \n",
       "3  B007P5S2JY            Clothing, Shoes & Jewelry\u0001Women\u0001Clothing\u0001Pants...   \n",
       "4  B007TUODIE            Clothing, Shoes & Jewelry\u0001Women\u0001Clothing\u0001Dress...   \n",
       "5  B008EYPQFI               Clothing, Shoes & Jewelry\u0001Women\u0001Clothing\u0001Jeans   \n",
       "6  B008VO5UES  D.E.P.T.  Clothing, Shoes & Jewelry\u0001Women\u0001Clothing\u0001Coats...   \n",
       "7  B009O90116            Clothing, Shoes & Jewelry\u0001Women\u0001Clothing\u0001Tops,...   \n",
       "8  B00A85Z7MI   G2 Chic  Clothing, Shoes & Jewelry\u0001Women\u0001Clothing\u0001Tops,...   \n",
       "9  B00AEVHJZY   G2 Chic  Clothing, Shoes & Jewelry\u0001Women\u0001Clothing\u0001Tops,...   \n",
       "\n",
       "   category_levels                                        description  \\\n",
       "0                5                                                      \n",
       "1                5                                                      \n",
       "2                5                                                      \n",
       "3                5  Self fabric waistband with smooth interior ela...   \n",
       "4                5                                                      \n",
       "5                4  Look one size smaller in this Not Your Daughte...   \n",
       "6                5  The classic military inspired coat is updated ...   \n",
       "7                5                                                      \n",
       "8                5                                                      \n",
       "9                5  Round neck, oversized, quarter-sleeves, bumpy ...   \n",
       "\n",
       "              price                                              title  \\\n",
       "0   $21.59 - $26.99   PattyBoutik Women's Off Shoulder Long Sleeve Top   \n",
       "1                             AK Anne Klein Women's Ruffle Trench Coat   \n",
       "2                    XOXO Juniors Smock Bottom Embellished Printed ...   \n",
       "3                                             PUMA Women's Velour Pant   \n",
       "4                                             MYNE Women's Gemma Dress   \n",
       "5  $37.81 - $158.00        NYDJ Women's Petite Size Alina Legging Jean   \n",
       "6                                       D.E.P.T. Women's Fabulous Coat   \n",
       "7                    Southpole Juniors' Lightweight Lace-Fabric Hen...   \n",
       "8                        G2 Chic Women's Crochet Accent Cape Loose Top   \n",
       "9                    G2 Chic Women's Lightweight Solid Patterned La...   \n",
       "\n",
       "                                    url  \\\n",
       "0  https://www.amazon.com/dp/B0069J1Y2Y   \n",
       "1  https://www.amazon.com/dp/B006MQR4QY   \n",
       "2  https://www.amazon.com/dp/B0078LJ5KU   \n",
       "3  https://www.amazon.com/dp/B007P5S2JY   \n",
       "4  https://www.amazon.com/dp/B007TUODIE   \n",
       "5  https://www.amazon.com/dp/B008EYPQFI   \n",
       "6  https://www.amazon.com/dp/B008VO5UES   \n",
       "7  https://www.amazon.com/dp/B009O90116   \n",
       "8  https://www.amazon.com/dp/B00A85Z7MI   \n",
       "9  https://www.amazon.com/dp/B00AEVHJZY   \n",
       "\n",
       "                                               image  \n",
       "0  https://images-na.ssl-images-amazon.com/images...  \n",
       "1  https://images-na.ssl-images-amazon.com/images...  \n",
       "2  https://images-na.ssl-images-amazon.com/images...  \n",
       "3  https://images-na.ssl-images-amazon.com/images...  \n",
       "4  https://images-na.ssl-images-amazon.com/images...  \n",
       "5  https://images-na.ssl-images-amazon.com/images...  \n",
       "6  https://images-na.ssl-images-amazon.com/images...  \n",
       "7  https://images-na.ssl-images-amazon.com/images...  \n",
       "8  https://images-na.ssl-images-amazon.com/images...  \n",
       "9  https://images-na.ssl-images-amazon.com/images...  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "product_5core.limit(10).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa5a0537-3fd7-401f-b3e3-3bdcd122a4e7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "c3114401-5b1f-44f8-bc6a-36b4b52202c9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:40:21.601283Z",
     "iopub.status.busy": "2022-10-13T10:40:21.600842Z",
     "iopub.status.idle": "2022-10-13T10:40:23.592519Z",
     "shell.execute_reply": "2022-10-13T10:40:23.591869Z",
     "shell.execute_reply.started": "2022-10-13T10:40:21.601263Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "# user_5core.repartition(1).write.option(\"header\", True).csv(user_path, mode='overwrite')\n",
    "user_5core.repartition(1).write.parquet(user_path, mode='overwrite')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "8fbfa921-cb08-4ae1-ac67-ef7355f15798",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:40:23.593611Z",
     "iopub.status.busy": "2022-10-13T10:40:23.593380Z",
     "iopub.status.idle": "2022-10-13T10:40:26.056321Z",
     "shell.execute_reply": "2022-10-13T10:40:26.055526Z",
     "shell.execute_reply.started": "2022-10-13T10:40:23.593591Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "# product_5core.repartition(1).write.option(\"header\", True).csv(item_path, mode='overwrite')\n",
    "product_5core.repartition(1).write.parquet(item_path, mode='overwrite')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "3e2fec39-6a28-418d-b450-d4cf43bdd53c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:40:26.057628Z",
     "iopub.status.busy": "2022-10-13T10:40:26.057385Z",
     "iopub.status.idle": "2022-10-13T10:40:28.093562Z",
     "shell.execute_reply": "2022-10-13T10:40:28.092668Z",
     "shell.execute_reply.started": "2022-10-13T10:40:26.057606Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "# review_5core.repartition(1).write.option(\"header\", True).csv(interaction_path, mode='overwrite')\n",
    "review_5core.repartition(1).write.parquet(interaction_path, mode='overwrite')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "c14ad44e-30cc-4824-ac9a-73a462f1b973",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:40:28.095249Z",
     "iopub.status.busy": "2022-10-13T10:40:28.094719Z",
     "iopub.status.idle": "2022-10-13T10:40:28.765034Z",
     "shell.execute_reply": "2022-10-13T10:40:28.764181Z",
     "shell.execute_reply.started": "2022-10-13T10:40:28.095225Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                           PRE amazon_fashion_interaction.100K.extra.parquet/\n",
      "                           PRE amazon_fashion_interaction.100K.parquet/\n",
      "                           PRE amazon_fashion_interaction.csv/\n",
      "                           PRE amazon_fashion_interaction.parquet/\n",
      "                           PRE amazon_fashion_interaction.small.parquet/\n",
      "                           PRE amazon_fashion_item.100K.extra.parquet/\n",
      "                           PRE amazon_fashion_item.100K.parquet/\n",
      "                           PRE amazon_fashion_item.csv/\n",
      "                           PRE amazon_fashion_item.parquet/\n",
      "                           PRE amazon_fashion_item.small.parquet/\n",
      "                           PRE amazon_fashion_user.100K.extra.parquet/\n",
      "                           PRE amazon_fashion_user.100K.parquet/\n",
      "                           PRE amazon_fashion_user.csv/\n",
      "                           PRE amazon_fashion_user.parquet/\n",
      "                           PRE amazon_fashion_user.small.parquet/\n",
      "2022-09-08 14:11:37 3247339254 amazon_fashion_product.json\n",
      "2022-09-08 14:13:55 3059298908 amazon_fashion_review.json\n"
     ]
    }
   ],
   "source": [
    "!aws s3 ls ${MY_S3_BUCKET}/sass/ecommerce/amazonfashion/dataset/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "80eb02aa-95ab-4e95-bb89-506d00672a3f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:40:28.766668Z",
     "iopub.status.busy": "2022-10-13T10:40:28.766133Z",
     "iopub.status.idle": "2022-10-13T10:40:28.769572Z",
     "shell.execute_reply": "2022-10-13T10:40:28.768950Z",
     "shell.execute_reply.started": "2022-10-13T10:40:28.766645Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# product_5core.filter(F.col('item_id') == 'B018448KC8').show(1,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "a7b20db5-0e72-482d-b433-cf37aa34d9f7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-13T10:40:28.770506Z",
     "iopub.status.busy": "2022-10-13T10:40:28.770314Z",
     "iopub.status.idle": "2022-10-13T10:40:31.512376Z",
     "shell.execute_reply": "2022-10-13T10:40:31.511701Z",
     "shell.execute_reply.started": "2022-10-13T10:40:28.770489Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Debug -- spark stop\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "22/10/13 18:40:28 WARN ExecutorPodsWatchSnapshotSource: Kubernetes client has been closed (this is expected if the application is shutting down.)\n"
     ]
    }
   ],
   "source": [
    "stop_spark(spark)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d154d29-9d91-4d05-b1bc-6c320e841fbe",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
